{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 利用XGboost实现Happy Customer Bank目标客户（贷款成功的客户）识别"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1 数据读取、处理及特征工程"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\hplxg\\Anaconda3\\envs\\python3\\lib\\site-packages\\sklearn\\cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "# 导入必要的工具包\n",
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "\n",
    "from scipy.special import cbrt\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.cross_validation import KFold\n",
    "from sklearn.cross_validation import StratifiedKFold\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.1 数据初步处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\hplxg\\Anaconda3\\envs\\python3\\lib\\site-packages\\IPython\\core\\interactiveshell.py:2728: DtypeWarning: Columns (18) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    }
   ],
   "source": [
    "# 读入数据\n",
    "dateparse = lambda x: pd.datetime.strptime(x, '%d-%b-%y')\n",
    "data = pd.read_csv('Train.csv', index_col = 'ID', encoding = 'latin1', parse_dates = ['Lead_Creation_Date'], date_parser=dateparse)\n",
    "test = pd.read_csv('Test.csv', index_col = 'ID', parse_dates = ['Lead_Creation_Date'], encoding = 'latin1', date_parser=dateparse)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 87020 entries, ID000002C20 to ID124821V10\n",
      "Data columns (total 25 columns):\n",
      "Gender                   87020 non-null object\n",
      "City                     86017 non-null object\n",
      "Monthly_Income           87020 non-null int64\n",
      "DOB                      87020 non-null object\n",
      "Lead_Creation_Date       87020 non-null datetime64[ns]\n",
      "Loan_Amount_Applied      86949 non-null float64\n",
      "Loan_Tenure_Applied      86949 non-null float64\n",
      "Existing_EMI             86949 non-null float64\n",
      "Employer_Name            86949 non-null object\n",
      "Salary_Account           75256 non-null object\n",
      "Mobile_Verified          87020 non-null object\n",
      "Var5                     87020 non-null int64\n",
      "Var1                     87019 non-null object\n",
      "Loan_Amount_Submitted    52407 non-null float64\n",
      "Loan_Tenure_Submitted    52407 non-null float64\n",
      "Interest_Rate            27726 non-null float64\n",
      "Processing_Fee           27420 non-null float64\n",
      "EMI_Loan_Submitted       27727 non-null object\n",
      "Filled_Form              87020 non-null object\n",
      "Device_Type              87020 non-null object\n",
      "Var2                     87020 non-null object\n",
      "Source                   87020 non-null object\n",
      "Var4                     87020 non-null int64\n",
      "LoggedIn                 87020 non-null int64\n",
      "Disbursed                87019 non-null float64\n",
      "dtypes: datetime64[ns](1), float64(8), int64(4), object(12)\n",
      "memory usage: 17.3+ MB\n"
     ]
    }
   ],
   "source": [
    "data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 37717 entries, ID000026A10 to ID124823X30\n",
      "Data columns (total 23 columns):\n",
      "Gender                   37717 non-null object\n",
      "City                     37319 non-null object\n",
      "Monthly_Income           37717 non-null int64\n",
      "DOB                      37717 non-null object\n",
      "Lead_Creation_Date       37717 non-null datetime64[ns]\n",
      "Loan_Amount_Applied      37677 non-null float64\n",
      "Loan_Tenure_Applied      37677 non-null float64\n",
      "Existing_EMI             37677 non-null float64\n",
      "Employer_Name            37675 non-null object\n",
      "Salary_Account           32680 non-null object\n",
      "Mobile_Verified          37717 non-null object\n",
      "Var5                     37717 non-null int64\n",
      "Var1                     37717 non-null object\n",
      "Loan_Amount_Submitted    22795 non-null float64\n",
      "Loan_Tenure_Submitted    22795 non-null float64\n",
      "Interest_Rate            12110 non-null float64\n",
      "Processing_Fee           11971 non-null float64\n",
      "EMI_Loan_Submitted       12110 non-null float64\n",
      "Filled_Form              37717 non-null object\n",
      "Device_Type              37717 non-null object\n",
      "Var2                     37717 non-null object\n",
      "Source                   37717 non-null object\n",
      "Var4                     37717 non-null int64\n",
      "dtypes: datetime64[ns](1), float64(8), int64(3), object(11)\n",
      "memory usage: 6.9+ MB\n"
     ]
    }
   ],
   "source": [
    "test.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 可以观察到训练集和测试集的两组数据都没有缺失值，测试集数据比训练集数据少两个特征，LoggedIn和Disbursed。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Gender</th>\n",
       "      <th>City</th>\n",
       "      <th>Monthly_Income</th>\n",
       "      <th>DOB</th>\n",
       "      <th>Lead_Creation_Date</th>\n",
       "      <th>Loan_Amount_Applied</th>\n",
       "      <th>Loan_Tenure_Applied</th>\n",
       "      <th>Existing_EMI</th>\n",
       "      <th>Employer_Name</th>\n",
       "      <th>Salary_Account</th>\n",
       "      <th>...</th>\n",
       "      <th>Interest_Rate</th>\n",
       "      <th>Processing_Fee</th>\n",
       "      <th>EMI_Loan_Submitted</th>\n",
       "      <th>Filled_Form</th>\n",
       "      <th>Device_Type</th>\n",
       "      <th>Var2</th>\n",
       "      <th>Source</th>\n",
       "      <th>Var4</th>\n",
       "      <th>LoggedIn</th>\n",
       "      <th>Disbursed</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ID000002C20</th>\n",
       "      <td>Female</td>\n",
       "      <td>Delhi</td>\n",
       "      <td>20000</td>\n",
       "      <td>23-May-78</td>\n",
       "      <td>2015-05-15</td>\n",
       "      <td>300000.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>CYBOSOL</td>\n",
       "      <td>HDFC Bank</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>N</td>\n",
       "      <td>Web-browser</td>\n",
       "      <td>G</td>\n",
       "      <td>S122</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID000004E40</th>\n",
       "      <td>Male</td>\n",
       "      <td>Mumbai</td>\n",
       "      <td>35000</td>\n",
       "      <td>7-Oct-85</td>\n",
       "      <td>2015-05-04</td>\n",
       "      <td>200000.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>TATA CONSULTANCY SERVICES LTD (TCS)</td>\n",
       "      <td>ICICI Bank</td>\n",
       "      <td>...</td>\n",
       "      <td>13.25</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6762.9</td>\n",
       "      <td>N</td>\n",
       "      <td>Web-browser</td>\n",
       "      <td>G</td>\n",
       "      <td>S122</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID000007H20</th>\n",
       "      <td>Male</td>\n",
       "      <td>Panchkula</td>\n",
       "      <td>22500</td>\n",
       "      <td>10-Oct-81</td>\n",
       "      <td>2015-05-19</td>\n",
       "      <td>600000.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>ALCHEMIST HOSPITALS LTD</td>\n",
       "      <td>State Bank of India</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>N</td>\n",
       "      <td>Web-browser</td>\n",
       "      <td>B</td>\n",
       "      <td>S143</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID000008I30</th>\n",
       "      <td>Male</td>\n",
       "      <td>Saharsa</td>\n",
       "      <td>35000</td>\n",
       "      <td>30-Nov-87</td>\n",
       "      <td>2015-05-09</td>\n",
       "      <td>1000000.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>BIHAR GOVERNMENT</td>\n",
       "      <td>State Bank of India</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>N</td>\n",
       "      <td>Web-browser</td>\n",
       "      <td>B</td>\n",
       "      <td>S143</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID000009J40</th>\n",
       "      <td>Male</td>\n",
       "      <td>Bengaluru</td>\n",
       "      <td>100000</td>\n",
       "      <td>17-Feb-84</td>\n",
       "      <td>2015-05-20</td>\n",
       "      <td>500000.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>25000.0</td>\n",
       "      <td>GLOBAL EDGE SOFTWARE</td>\n",
       "      <td>HDFC Bank</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>N</td>\n",
       "      <td>Web-browser</td>\n",
       "      <td>B</td>\n",
       "      <td>S134</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             Gender       City  Monthly_Income        DOB Lead_Creation_Date  \\\n",
       "ID                                                                             \n",
       "ID000002C20  Female      Delhi           20000  23-May-78         2015-05-15   \n",
       "ID000004E40    Male     Mumbai           35000   7-Oct-85         2015-05-04   \n",
       "ID000007H20    Male  Panchkula           22500  10-Oct-81         2015-05-19   \n",
       "ID000008I30    Male    Saharsa           35000  30-Nov-87         2015-05-09   \n",
       "ID000009J40    Male  Bengaluru          100000  17-Feb-84         2015-05-20   \n",
       "\n",
       "             Loan_Amount_Applied  Loan_Tenure_Applied  Existing_EMI  \\\n",
       "ID                                                                    \n",
       "ID000002C20             300000.0                  5.0           0.0   \n",
       "ID000004E40             200000.0                  2.0           0.0   \n",
       "ID000007H20             600000.0                  4.0           0.0   \n",
       "ID000008I30            1000000.0                  5.0           0.0   \n",
       "ID000009J40             500000.0                  2.0       25000.0   \n",
       "\n",
       "                                   Employer_Name       Salary_Account  \\\n",
       "ID                                                                      \n",
       "ID000002C20                              CYBOSOL            HDFC Bank   \n",
       "ID000004E40  TATA CONSULTANCY SERVICES LTD (TCS)           ICICI Bank   \n",
       "ID000007H20              ALCHEMIST HOSPITALS LTD  State Bank of India   \n",
       "ID000008I30                     BIHAR GOVERNMENT  State Bank of India   \n",
       "ID000009J40                 GLOBAL EDGE SOFTWARE            HDFC Bank   \n",
       "\n",
       "               ...    Interest_Rate  Processing_Fee EMI_Loan_Submitted  \\\n",
       "ID             ...                                                       \n",
       "ID000002C20    ...              NaN             NaN                NaN   \n",
       "ID000004E40    ...            13.25             NaN             6762.9   \n",
       "ID000007H20    ...              NaN             NaN                NaN   \n",
       "ID000008I30    ...              NaN             NaN                NaN   \n",
       "ID000009J40    ...              NaN             NaN                NaN   \n",
       "\n",
       "             Filled_Form  Device_Type  Var2  Source Var4 LoggedIn Disbursed  \n",
       "ID                                                                           \n",
       "ID000002C20            N  Web-browser     G    S122    1        0       0.0  \n",
       "ID000004E40            N  Web-browser     G    S122    3        0       0.0  \n",
       "ID000007H20            N  Web-browser     B    S143    1        0       0.0  \n",
       "ID000008I30            N  Web-browser     B    S143    3        0       0.0  \n",
       "ID000009J40            N  Web-browser     B    S134    3        1       0.0  \n",
       "\n",
       "[5 rows x 25 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Gender</th>\n",
       "      <th>City</th>\n",
       "      <th>Monthly_Income</th>\n",
       "      <th>DOB</th>\n",
       "      <th>Lead_Creation_Date</th>\n",
       "      <th>Loan_Amount_Applied</th>\n",
       "      <th>Loan_Tenure_Applied</th>\n",
       "      <th>Existing_EMI</th>\n",
       "      <th>Employer_Name</th>\n",
       "      <th>Salary_Account</th>\n",
       "      <th>...</th>\n",
       "      <th>Loan_Amount_Submitted</th>\n",
       "      <th>Loan_Tenure_Submitted</th>\n",
       "      <th>Interest_Rate</th>\n",
       "      <th>Processing_Fee</th>\n",
       "      <th>EMI_Loan_Submitted</th>\n",
       "      <th>Filled_Form</th>\n",
       "      <th>Device_Type</th>\n",
       "      <th>Var2</th>\n",
       "      <th>Source</th>\n",
       "      <th>Var4</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ID000026A10</th>\n",
       "      <td>Male</td>\n",
       "      <td>Dehradun</td>\n",
       "      <td>21500</td>\n",
       "      <td>3-Apr-87</td>\n",
       "      <td>2015-05-05</td>\n",
       "      <td>100000.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>APTARA INC</td>\n",
       "      <td>ICICI Bank</td>\n",
       "      <td>...</td>\n",
       "      <td>100000.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>1000.0</td>\n",
       "      <td>2649.39</td>\n",
       "      <td>N</td>\n",
       "      <td>Web-browser</td>\n",
       "      <td>B</td>\n",
       "      <td>S122</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID000054C40</th>\n",
       "      <td>Male</td>\n",
       "      <td>Mumbai</td>\n",
       "      <td>42000</td>\n",
       "      <td>12-May-80</td>\n",
       "      <td>2015-05-01</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>ATUL LTD</td>\n",
       "      <td>Axis Bank</td>\n",
       "      <td>...</td>\n",
       "      <td>690000.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>13800.0</td>\n",
       "      <td>19849.90</td>\n",
       "      <td>Y</td>\n",
       "      <td>Mobile</td>\n",
       "      <td>C</td>\n",
       "      <td>S133</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID000066O10</th>\n",
       "      <td>Female</td>\n",
       "      <td>Jaipur</td>\n",
       "      <td>10000</td>\n",
       "      <td>19-Sep-89</td>\n",
       "      <td>2015-05-01</td>\n",
       "      <td>300000.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>SHAREKHAN PVT LTD</td>\n",
       "      <td>ICICI Bank</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>N</td>\n",
       "      <td>Web-browser</td>\n",
       "      <td>B</td>\n",
       "      <td>S133</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID000110G00</th>\n",
       "      <td>Female</td>\n",
       "      <td>Chennai</td>\n",
       "      <td>14650</td>\n",
       "      <td>15-Aug-91</td>\n",
       "      <td>2015-05-01</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>MAERSK GLOBAL SERVICE CENTRES</td>\n",
       "      <td>HDFC Bank</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>N</td>\n",
       "      <td>Mobile</td>\n",
       "      <td>C</td>\n",
       "      <td>S133</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID000113J30</th>\n",
       "      <td>Male</td>\n",
       "      <td>Chennai</td>\n",
       "      <td>23400</td>\n",
       "      <td>22-Jul-87</td>\n",
       "      <td>2015-05-01</td>\n",
       "      <td>100000.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5000.0</td>\n",
       "      <td>SCHAWK</td>\n",
       "      <td>Axis Bank</td>\n",
       "      <td>...</td>\n",
       "      <td>100000.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>N</td>\n",
       "      <td>Web-browser</td>\n",
       "      <td>B</td>\n",
       "      <td>S143</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 23 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             Gender      City  Monthly_Income        DOB Lead_Creation_Date  \\\n",
       "ID                                                                            \n",
       "ID000026A10    Male  Dehradun           21500   3-Apr-87         2015-05-05   \n",
       "ID000054C40    Male    Mumbai           42000  12-May-80         2015-05-01   \n",
       "ID000066O10  Female    Jaipur           10000  19-Sep-89         2015-05-01   \n",
       "ID000110G00  Female   Chennai           14650  15-Aug-91         2015-05-01   \n",
       "ID000113J30    Male   Chennai           23400  22-Jul-87         2015-05-01   \n",
       "\n",
       "             Loan_Amount_Applied  Loan_Tenure_Applied  Existing_EMI  \\\n",
       "ID                                                                    \n",
       "ID000026A10             100000.0                  3.0           0.0   \n",
       "ID000054C40                  0.0                  0.0           0.0   \n",
       "ID000066O10             300000.0                  2.0           0.0   \n",
       "ID000110G00                  0.0                  0.0           0.0   \n",
       "ID000113J30             100000.0                  1.0        5000.0   \n",
       "\n",
       "                             Employer_Name Salary_Account ...   \\\n",
       "ID                                                        ...    \n",
       "ID000026A10                     APTARA INC     ICICI Bank ...    \n",
       "ID000054C40                       ATUL LTD      Axis Bank ...    \n",
       "ID000066O10              SHAREKHAN PVT LTD     ICICI Bank ...    \n",
       "ID000110G00  MAERSK GLOBAL SERVICE CENTRES      HDFC Bank ...    \n",
       "ID000113J30                         SCHAWK      Axis Bank ...    \n",
       "\n",
       "            Loan_Amount_Submitted  Loan_Tenure_Submitted Interest_Rate  \\\n",
       "ID                                                                       \n",
       "ID000026A10              100000.0                    3.0          20.0   \n",
       "ID000054C40              690000.0                    5.0          24.0   \n",
       "ID000066O10                   NaN                    NaN           NaN   \n",
       "ID000110G00                   NaN                    NaN           NaN   \n",
       "ID000113J30              100000.0                    2.0           NaN   \n",
       "\n",
       "             Processing_Fee  EMI_Loan_Submitted  Filled_Form  Device_Type  \\\n",
       "ID                                                                          \n",
       "ID000026A10          1000.0             2649.39            N  Web-browser   \n",
       "ID000054C40         13800.0            19849.90            Y       Mobile   \n",
       "ID000066O10             NaN                 NaN            N  Web-browser   \n",
       "ID000110G00             NaN                 NaN            N       Mobile   \n",
       "ID000113J30             NaN                 NaN            N  Web-browser   \n",
       "\n",
       "             Var2 Source Var4  \n",
       "ID                             \n",
       "ID000026A10     B   S122    3  \n",
       "ID000054C40     C   S133    5  \n",
       "ID000066O10     B   S133    1  \n",
       "ID000110G00     C   S133    1  \n",
       "ID000113J30     B   S143    1  \n",
       "\n",
       "[5 rows x 23 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZUAAAEKCAYAAADaa8itAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAFpFJREFUeJzt3Xu0JWV95vHvw02Qi4C2LtNAGmNHJTreWoImEy8oAk5oZEGCyxEGmcFkAHESHTHJsuNdEoRI4iWsQLiMw0UGtEWUQQSdOIo0F8UGWXQwQg8EWkFEGGEafvPHfg/s7j6Xorv2OWzP97PWXqfqrbdq/7ar5VlVb9VbqSokSerDZnNdgCTpV4ehIknqjaEiSeqNoSJJ6o2hIknqjaEiSeqNoSJJ6o2hIknqjaEiSerNFnNdwGx7xjOeUYsWLZrrMiRpbFxzzTU/qaoFXfrOu1BZtGgRK1asmOsyJGlsJPlx175e/pIk9cZQkST1xlCRJPXGUJEk9cZQkST1xlCRJPXGUJEk9cZQkST1xlCRJPVm3j1Rv6le/p6z5roEPQld89eHzXUJ0pOCZyqSpN4YKpKk3hgqkqTeGCqSpN4YKpKk3hgqkqTeGCqSpN4YKpKk3hgqkqTejDRUkvyXJCuT/CDJOUm2TrJ7kquS3JLkvCRbtb5Paeur2vZFQ8d5X2u/Ockbh9r3bW2rkhw/yt8iSZrZyEIlyULgncCSqnohsDlwKHACcHJVLQbuBY5suxwJ3FtVzwVObv1Iskfb77eAfYFPJ9k8yebAp4D9gD2At7S+kqQ5MurLX1sA2yTZAngqcCfwOuCCtv1M4MC2vLSt07bvnSSt/dyqeqiqfgSsAvZsn1VVdWtVPQyc2/pKkubIyEKlqv4PcCJwG4MwuQ+4BvhZVa1t3VYDC9vyQuD2tu/a1v/pw+3r7TNVuyRpjozy8tdODM4cdgd+DdiWwaWq9dXELlNse6Ltk9VyVJIVSVasWbNmptIlSRtplJe/Xg/8qKrWVNX/Ay4EXgXs2C6HAewC3NGWVwO7ArTtTwPuGW5fb5+p2jdQVadW1ZKqWrJgwYI+fpskaRKjDJXbgL2SPLWNjewN3AhcARzc+hwOfLEtL2/rtO1fr6pq7Ye2u8N2BxYD3wWuBha3u8m2YjCYv3yEv0eSNIORvaSrqq5KcgFwLbAWuA44FfgycG6SD7e209oupwFnJ1nF4Azl0HaclUnOZxBIa4Gjq+oRgCTHAJcyuLPs9KpaOarfI0ma2Ujf/FhVy4Bl6zXfyuDOrfX7/hI4ZIrjfAT4yCTtlwCXbHqlkqQ++ES9JKk3hookqTeGiiSpN4aKJKk3hookqTeGiiSpN4aKJKk3hookqTeGiiSpN4aKJKk3hookqTeGiiSpN4aKJKk3hookqTeGiiSpN4aKJKk3hookqTeGiiSpN4aKJKk3hookqTeGiiSpN4aKJKk3M4ZKkm2TbNaWfzPJAUm2HH1pkqRx0+VM5ZvA1kkWApcDRwBnjLIoSdJ46hIqqaoHgYOAv62qNwN7jLYsSdI46hQqSV4JvBX4cmvbYnQlSZLGVZdQeRfwPuCiqlqZ5DnAFaMtS5I0jmY846iqbwDfSLJtW78VeOeoC5MkjZ8ud3+9MsmNwE1t/cVJPj3yyiRJY6fL5a+/Ad4I/BSgqr4H/N4oi5IkjadODz9W1e3rNT0yglokSWOuy11ctyd5FVBJtmIwnnLTaMuSJI2jLmcqfwQcDSwEVgMvaeuSJK2jy91fP2HwjIokSdPqcvfXmUl2HFrfKcnpoy1LkjSOulz++jdV9bOJlaq6F3jp6EqSJI2rLqGyWZKdJlaS7IzTtEiSJtElHD4B/O8kF7T1Q4CPjK4kSdK46jJQf1aSa4DXAgEOqqobR16ZJGnsdH3z4w+BC4EvAr9IsluXnZLsmOSCJD9MclOb8mXnJJcluaX93an1TZJTkqxK8v0kLxs6zuGt/y1JDh9qf3mSG9o+pyRJ958uSepbl7u/jgXuAi4DLmYw/f3FHY//SeCrVfV84MUMHpo8Hri8qhYzeOnX8a3vfsDi9jkK+Ez7/p2BZcBvA3sCy4bGeD7T+k7st2/HuiRJI9BlTOU44HlV9dMncuAkOzCYI+w/AFTVw8DDSZYCr2ndzgSuBN4LLAXOqqoCvtPOcp7d+l5WVfe0414G7JvkSmCHqvp2az8LOBD4yhOpU5LUny6Xv24H7tuIYz8HWAP8Y5LrkvxDmz7/WVV1J0D7+8zWf2H7rgmrW9t07asnaZckzZEuZyq3Alcm+TLw0ERjVZ3U4dgvA46tqquSfJLHL3VNZrLxkNqI9g0PnBzF4DIZu+3WaThIkrQRupyp3MZgPGUrYPuhz0xWA6ur6qq2fgGDkLmrXdai/b17qP+uQ/vvAtwxQ/suk7RvoKpOraolVbVkwYIFHUqXJG2MLrcUfwAgybZV9UDXA1fVvya5PcnzqupmYG/gxvY5HPh4+/vFtsty4Jgk5zIYlL+vqu5Mcinw0aHB+X2A91XVPUnuT7IXcBVwGPC3XeuTJPVvxlBJ8krgNGA7YLckLwbeUVX/ucPxjwU+16bMvxU4gsHZ0flJjmRwFnRI63sJsD+wCniw9aWFx4eAq1u/D04M2gN/DJwBbMNggN5BekmaQ13GVCbe/LgcBm9+TNLpzY9VdT2wZJJNe0/St5hiSv2qOh3YYBLLqloBvLBLLZKk0fPNj5Kk3vjmR0lSb3zzoySpN9OeqSTZHHhbVfnmR0nSjKY9U6mqRxhMnyJJ0oy6jKl8K8nfAecBjz2nUlXXjqwqSdJY6hIqr2p/PzjUVsDr+i9HkjTOZhpT2Qz4TFWdP0v1SJLG2ExjKo8Cx8xSLZKkMdflluLLkrw7ya7trY07txdnSZK0ji5jKm9vf4efTSkG70uRJOkxXWYp3n02CpEkjb8usxQfNll7VZ3VfzmSpHHW5fLXK4aWt2Yww/C1gKEiSVpHl8tfxw6vJ3kacPbIKpIkja1OU9+v50Fgcd+FSJLGX5cxlS8xuNsLBiG0B+DDkJKkDXQZUzlxaHkt8OOqWj2ieiRJY6xLqNwG3FlVvwRIsk2SRVX1LyOtTJI0drqMqXweeHRo/ZHWJknSOrqEyhZV9fDESlveanQlSZLGVZdQWZPkgImVJEuBn4yuJEnSuOoypvJHwOfai7pg8J76SZ+ylyTNb10efvxnYK8k2wGpqvtHX5YkaRzNePkryUeT7FhVv6iq+5PslOTDs1GcJGm8dBlT2a+qfjaxUlX3AvuPriRJ0rjqEiqbJ3nKxEqSbYCnTNNfkjRPdRmo/2/A5Un+kcF0LW8HzhxpVZKksdRloP6vknwfeH1r+lBVXTrasiRJ46jLmQrAdcCWDM5UrhtdOZKkcdbl7q8/AL4LHAz8AXBVkoNHXZgkafx0OVP5c+AVVXU3QJIFwNeAC0ZZmCRp/HS5+2uziUBpftpxP0nSPNPlTOWrSS4FzmnrfwhcMrqSJEnjqsvdX+9JchDwu0CAU6vqopFXJkkaO53u/qqqC4ELR1yLJGnMOTYiSeqNoSJJ6s2UoZLk8vb3hNkrR5I0zqYbU3l2klcDByQ5l8Eg/WOq6tqRViZJGjvTXf56P3A8sAtwEvCJoc+JXb8gyeZJrktycVvfPclVSW5Jcl6SrVr7U9r6qrZ90dAx3tfab07yxqH2fVvbqiTHd//ZkqRRmDJUquqCqtoP+Kuqeu16n9c9ge84DrhpaP0E4OSqWgzcCxzZ2o8E7q2q5wInt34k2QM4FPgtYF/g0y2oNgc+BewH7AG8pfWVJM2RGQfqq+pDSQ5IcmL7/LuuB0+yC/Am4B/aeoDX8fgUL2cCB7blpTw+pf4FwN6t/1Lg3Kp6qKp+BKwC9myfVVV1a1U9DJzb+kqS5kiXCSU/xuBs48b2Oa61dfE3wH8FHm3rTwd+VlVr2/pqYGFbXgjcDtC239f6P9a+3j5TtU/2G45KsiLJijVr1nQsXZL0RHW5pfhNwBuq6vSqOp3BJag3zbRTO6O5u6quGW6epGvNsO2Jtm/YWHVqVS2pqiULFiyYpmpJ0qbo+j6VHYF72vLTOu7zOwzuHNsf2BrYgcGZy45JtmhnI7sAd7T+q4FdgdVJtmjfc89Q+4ThfaZqlyTNgS5nKh8DrktyRpIzgWuAj860U1W9r6p2qapFDAbav15VbwWuYPBuFoDDgS+25eVtnbb961VVrf3QdnfY7sBiBu93uRpY3O4m26p9x/IOv0eSNCJdJpQ8J8mVwCsYXHJ6b1X96yZ853uBc5N8mMFbJE9r7acBZydZxeAM5dD2/SuTnM9gPGctcHRVPQKQ5BjgUmBz4PSqWrkJdUmSNlHXCSXvZBPOAqrqSuDKtnwrgzu31u/zS+CQKfb/CPCRSdovwWn4JelJw7m/JEm9MVQkSb2ZNlSSbJbkB7NVjCRpvE0bKlX1KPC9JLvNUj2SpDHWZaD+2cDKJN8FHphorKoDRlaVJGksdQmVD4y8CknSr4Quz6l8I8mvA4ur6mtJnsrguRBJktbRZULJ/8Rg1uC/b00LgS+MsihJ0njqckvx0Qzm8fo5QFXdAjxzlEVJksZTl1B5qL2vBIA22eOkswFLkua3LqHyjSR/BmyT5A3A54EvjbYsSdI46hIqxwNrgBuAdzCYa+svRlmUJGk8dbn769E25f1VDC573dympJckaR0zhkqSNwGfBf6ZwdT3uyd5R1V9ZdTFSZLGS5eHHz8BvLaqVgEk+Q3gy4ChIklaR5cxlbsnAqW5Fbh7RPVIksbYlGcqSQ5qiyuTXAKcz2BM5RAGr/KVJGkd013++v2h5buAV7flNcBOI6tIkjS2pgyVqjpiNguRJI2/Lnd/7Q4cCywa7u/U95Kk9XW5++sLwGkMnqJ/dLTlSJLGWZdQ+WVVnTLySiRJY69LqHwyyTLgfwIPTTRW1bUjq0qSNJa6hMqLgLcBr+Pxy1/V1iVJekyXUHkz8Jzh6e8lSZpMlyfqvwfsOOpCJEnjr8uZyrOAHya5mnXHVLylWJK0ji6hsmzkVUiSfiV0eZ/KN2ajEEnS+OvyRP39PP5O+q2ALYEHqmqHURYmSRo/Xc5Uth9eT3IgsOfIKpIkja0ud3+to6q+gM+oSJIm0eXy10FDq5sBS3j8cpgkSY/pcvfX8HtV1gL/AiwdSTWSpLHWZUzF96pIkjqZ7nXC759mv6qqD42gHknSGJvuTOWBSdq2BY4Eng4YKpKkdUz3OuFPTCwn2R44DjgCOBf4xFT7SZLmr2nHVJLsDPwJ8FbgTOBlVXXvbBQmSRo/Uz6nkuSvgauB+4EXVdVfPpFASbJrkiuS3JRkZZLjWvvOSS5Lckv7u1NrT5JTkqxK8v0kLxs61uGt/y1JDh9qf3mSG9o+pyTJRvxvIEnqyXQPP/4p8GvAXwB3JPl5+9yf5Ocdjr0W+NOqegGwF3B0kj2A44HLq2oxcHlbB9gPWNw+RwGfgcfOlpYBv83gSf5lE0HU+hw1tN++3X62JGkUpgyVqtqsqrapqu2raoehz/Zd5v2qqjsnXjlcVfcDNwELGTzjcmbrdiZwYFteCpxVA98BdkzybOCNwGVVdU87U7oM2Ldt26Gqvl1VBZw1dCxJ0hx4wtO0bIwki4CXAlcBz6qqO2EQPMAzW7eFwO1Du61ubdO1r56kfbLvPyrJiiQr1qxZs6k/R5I0hZGHSpLtgP8BvKuqprtsNtl4SG1E+4aNVadW1ZKqWrJgwYKZSpYkbaSRhkqSLRkEyueq6sLWfFe7dEX7e3drXw3sOrT7LsAdM7TvMkm7JGmOjCxU2p1YpwE3VdVJQ5uWAxN3cB0OfHGo/bB2F9hewH3t8tilwD5JdmoD9PsAl7Zt9yfZq33XYUPHkiTNgS4TSm6s3wHeBtyQ5PrW9mfAx4HzkxwJ3AYc0rZdAuwPrAIeZPCgJVV1T5IPMbi9GeCDVXVPW/5j4AxgG+Ar7SNJmiMjC5Wq+icmH/cA2HuS/gUcPcWxTgdOn6R9BfDCTShTktSjWbn7S5I0PxgqkqTeGCqSpN4YKpKk3hgqkqTeGCqSpN4YKpKk3hgqkqTeGCqSpN4YKpKk3hgqkqTeGCqSpN4YKpKk3hgqkqTeGCqSpN4YKpKk3hgqkqTeGCqSpN4YKpKk3hgqkqTeGCqSpN4YKpKk3hgqkqTeGCqSpN4YKpKk3hgqkqTeGCqSpN4YKpKk3hgqkqTeGCqSpN4YKpKk3hgqkqTeGCqSpN4YKpKk3hgqkqTeGCqSpN4YKpKk3hgqkqTeGCqSpN6Mfagk2TfJzUlWJTl+ruuRpPlsi7kuYFMk2Rz4FPAGYDVwdZLlVXXj3FYmzY3bPviiuS5BT0K7vf+GWfuucT9T2RNYVVW3VtXDwLnA0jmuSZLmrXEPlYXA7UPrq1ubJGkOjPXlLyCTtNUGnZKjgKPa6i+S3DzSquaPZwA/mesingxy4uFzXYI25L/PCcsm+0/lE/LrXTuOe6isBnYdWt8FuGP9TlV1KnDqbBU1XyRZUVVL5roOaTL++5wb437562pgcZLdk2wFHAosn+OaJGneGuszlapam+QY4FJgc+D0qlo5x2VJ0rw11qECUFWXAJfMdR3zlJcU9WTmv885kKoNxrUlSdoo4z6mIkl6EjFUNKOZpsJJ8pQk57XtVyVZNPtVaj5KcnqSu5P8YIrtSXJK+7f5/SQvm+0a5xtDRdMamgpnP2AP4C1J9liv25HAvVX1XOBk4ITZrVLz2BnAvtNs3w9Y3D5HAZ+ZhZrmNUNFM+kyFc5S4My2fAGwd5JNftpKmklVfRO4Z5ouS4GzauA7wI5Jnj071c1Phopm0mUqnMf6VNVa4D7g6bNSnTQ9p3KaZYaKZtJlKpxO0+VIc8B/m7PMUNFMukyF81ifJFsAT2P6SxLSbOk0lZP6Y6hoJl2mwlkOTMyoeDDw9fIBKD05LAcOa3eB7QXcV1V3znVRv8rG/ol6jdZUU+Ek+SCwoqqWA6cBZydZxeAM5dC5q1jzSZJzgNcAz0iyGlgGbAlQVZ9lMNvG/sAq4EHgiLmpdP7wiXpJUm+8/CVJ6o2hIknqjaEiSeqNoSJJ6o2hIknqjaEidZTkkSTXJ1mZ5HtJ/iTJZm3bkiSnTLPva5JcPHvVbvD9f5nk3XP1/Zo/fE5F6u7/VtVLAJI8E/jvDGYPWFZVK4AVo/riJFu0edWkJzXPVKSNUFV3M5hK/Zj2tPZjZyJJXt3OaK5Pcl2S7dtuOyS5KMmNST47dJbzi4njJjk4yRlt+YwkJyW5AjhhquMmeU+Sq9v7Qj4wdKw/b+/B+RrwvNn430XyTEXaSFV1awuGZ6636d3A0VX1rSTbAb9s7XsyeCfNj4GvAgcxeFXAdH4TeH1VPZLkS+sfN8k+DN4VsieDyROXJ/k94AEGMxu8lMH/z68Frtm0XyzNzDMVadNMNgvut4CTkrwT2HHostV323tpHgHOAX63w/E/3/pPddx92uc6BsHxfAYh82+Bi6rqwar6ORvO1yaNhKEibaQkzwEeAe4ebq+qjwP/EdgG+E6S509sWu8QNUn71uv1eWCG4wb4WFW9pH2eW1WnTfF90sgZKtJGSLIA+Czwd+vPyJzkN6rqhqo6gcHg/USo7Nlme94M+EPgn1r7XUle0NrfPM13TnbcS4G3t8thJFnYbiL4JvDmJNu0sZff7+u3S9NxTEXqbpsk1zOYBXctcDZw0iT93pXktQzOYm4EvgK8Evg28HHgRQz+o39R6388cDGDNxT+ANhuiu/f4LhV9VCSFwDfbm9w/gXw76vq2iTnAdczGMP5X5vyw6WunKVYktQbL39JknpjqEiSemOoSJJ6Y6hIknpjqEiSemOoSJJ6Y6hIknpjqEiSevP/AQfaa5oYJqxDAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1c53d49bf60>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Disbursed 分布，查看各类样本是否均衡\n",
    "sns.countplot(data.Disbursed);\n",
    "pyplot.xlabel('Disbursed');\n",
    "pyplot.ylabel('Number of occurrences');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 通过观察train.csv 和 test.csv中的数据，以及由题目的字段说明，我们可知，Gender、City、Employer_Name、Salary_Account、Mobile_Verified、Var1、Filled_Form、Device_Type、Var2、Source这些特征的数据，都属于非数值型的分类数据，需要对这些特征的数据进行编码。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.2 特征工程"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 将 Lead_Creation_Date & DOB 的日期格式转化为整型的数字\n",
    "def fixDates(data):\n",
    "    data['DOB_yr'] = [item.split('-')[2] for item in data['DOB']]\n",
    "    data.DOB_yr = '19' + data.DOB_yr\n",
    "    data['DOB_mon'] = [item.split('-')[1] for item in data['DOB']]\n",
    "    data['DOB_day'] = [item.split('-')[0] for item in data['DOB']]\n",
    "    data.DOB = data.apply(lambda x: pd.datetime.strptime(\"{0} {1} {2} 00:00:00\".format(x['DOB_yr'],x['DOB_mon'], x['DOB_day']), \"%Y %b %d %H:%M:%S\"),axis=1)\n",
    "\n",
    "    # drop extra features\n",
    "    data.drop( [ 'DOB_mon', 'DOB_day' ] , axis=1, inplace=True) # 'DOB_yr', \n",
    "    data.DOB_yr = [int(x) for x in data.DOB_yr]\n",
    "\n",
    "    # convert dates to ordinal\n",
    "    data['Lead_Creation_Date'] = data['Lead_Creation_Date'].apply(lambda x: x.toordinal())    \n",
    "    data['DOB'] = data['DOB'].apply(lambda x: x.toordinal())  \n",
    "     \n",
    "    \n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dictMap(listOfMajors, non_major):\n",
    "    mapped_dict = {}\n",
    "    for i, major in enumerate(reversed(listOfMajors)):\n",
    "        mapped_dict[major] = (i+1)\n",
    "    mapped_dict[non_major] = 0\n",
    "    return mapped_dict\n",
    "\n",
    "def dictMap0(listOfMajors):\n",
    "    mapped_dict = {}\n",
    "    for i, major in enumerate(reversed(listOfMajors)):\n",
    "        mapped_dict[major] = i\n",
    "    return mapped_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 将LoggIn和Disbursed两个目标变量添加到test\n",
    "test['LoggedIn'] = 9999\n",
    "test['Disbursed'] = 9999\n",
    "    \n",
    "    # 对训练集和测试集数据进行集中处理\n",
    "combined = pd.concat( [ data, test ] )    \n",
    "    \n",
    "combined = fixDates(combined)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 处理训练集中的缺失值和空值,清楚错误值\n",
    "combined = combined.apply(lambda x: np.NaN if str(x).isspace() else x)\n",
    "combined.fillna(9999,inplace= True)\n",
    "combined['EMI_Loan_Submitted'] = combined['EMI_Loan_Submitted'].replace('N',0) # 替换这一列中的一个错误值N\n",
    "#combined = combined.dropna(axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    " # 将各个特征进行编码  \n",
    "    \n",
    "    # Gender - Female = 0, Male = 1\n",
    "combined['Gender'] = combined['Gender'].map( {'Female': 0, 'Male': 1} ).astype(int)\n",
    "    \n",
    "    # Filled_Form - N = 0, Y = 1  \n",
    "# combined['Filled_Form'] = combined['Filled_Form'].map( {'N': 0, 'Y': 1} ).astype(int)\n",
    "Filled_Form_le = LabelEncoder()\n",
    "Filled_Form_labels = Filled_Form_le.fit_transform(combined['Filled_Form'])\n",
    "combined['Filled_Form'] = Filled_Form_labels\n",
    "    \n",
    "    # Device_Type - Mobile = 0, Web-browser = 1\n",
    "# combined['Device_Type'] = combined['Device_Type'].map( {'Mobile': 0, 'Web-browser': 1} ).astype(int)\n",
    "Device_Type_le = LabelEncoder()\n",
    "Device_Type_labels = Device_Type_le.fit_transform(combined['Device_Type'])\n",
    "combined['Device_Type'] = Device_Type_labels\n",
    "    \n",
    "    # Mobile_Verified - N = 0, Y = 1\n",
    "#combined['Mobile_Verified'] = combined['Mobile_Verified'].map( {'N': 0, 'Y': 1} ).astype(int)\n",
    "Mobile_Verified_le = LabelEncoder()\n",
    "Mobile_Verified_labels = Mobile_Verified_le.fit_transform(combined['Mobile_Verified'])\n",
    "combined['Mobile_Verified'] = Mobile_Verified_labels\n",
    "    \n",
    "    # City\n",
    "city_counts = data.City.value_counts()\n",
    "major_cites = city_counts.index[:11]\n",
    "combined.loc[ ~combined['City'].isin(major_cites), 'City' ] = 'Non-major city'\n",
    "mapped_cities = dictMap(major_cites, 'Non-major city')\n",
    "combined['City'] = combined['City'].map( mapped_cities ).astype(int)\n",
    "    \n",
    "    # Employer_Name\n",
    "    # 清除不在TCS中的名字 \n",
    "data.loc[ data.Employer_Name.isin( ['TATA CONSALTANCY SERVICES', 'TATA CONSULTANCY SERVICE', 'TATA CONSULTANCY SERVICES', \n",
    "                                        'TATA CONSULTANCY SERVICES LIMITED', 'TATA CONSULTANCY SERVICES LTD (TCS)CONSUL'] ) , 'Employer_Name' ] = 'TATA CONSULTANCY SERVICES LTD (TCS)'\n",
    "combined.loc[ combined.Employer_Name.isin( ['TATA CONSALTANCY SERVICES', 'TATA CONSULTANCY SERVICE', 'TATA CONSULTANCY SERVICES', \n",
    "                                        'TATA CONSULTANCY SERVICES LIMITED', 'TATA CONSULTANCY SERVICES LTD (TCS)CONSUL'] ) , 'Employer_Name' ] = 'TATA CONSULTANCY SERVICES LTD (TCS)'\n",
    "     \n",
    "    # 按照Disbursed重新划分名字\n",
    "employer_groups = data.groupby('Employer_Name')['Disbursed'].sum()\n",
    "major_employers = list(employer_groups.sort_values()[-20:].index)\n",
    "major_employers.remove('0')\n",
    "major_employers.remove('TYPE SLOWLY FOR AUTO FILL')\n",
    "combined.loc[ ~combined['Employer_Name'].isin(major_employers), 'Employer_Name' ] = 'Non-major employer'\n",
    "    \n",
    "mapped_employers = dictMap(major_employers, 'Non-major employer')\n",
    "combined['Employer_Name'] = combined['Employer_Name'].map( mapped_employers ).astype(int)\n",
    "    \n",
    "    # Salary_Account\n",
    "bank_counts = data.Salary_Account.value_counts()\n",
    "major_banks = list(bank_counts.index[:20])\n",
    "combined.loc[ ~combined['Salary_Account'].isin(major_banks), 'Salary_Account' ] = 'Non-major bank'\n",
    "mapped_banks = dictMap(major_banks, 'Non-major bank')\n",
    "combined['Salary_Account'] = combined['Salary_Account'].map( mapped_banks ).astype(int)\n",
    "    \n",
    "    # Var1\n",
    "var1_counts = data.Var1.value_counts()\n",
    "major_var1 = list(var1_counts.index[:7]) # \n",
    "combined.loc[ ~combined['Var1'].isin(major_var1), 'Var1' ] = 'Non-major var1'\n",
    "mapped_var1 = dictMap(major_var1, 'Non-major var1')\n",
    "combined['Var1'] = combined['Var1'].map( mapped_var1 ).astype(int)\n",
    "    \n",
    "    # Var2\n",
    "var2_counts = data.Var2.value_counts()\n",
    "major_var2 = list(var2_counts.index)\n",
    "mapped_var2 = dictMap0(major_var2)\n",
    "combined['Var2'] = combined['Var2'].map( mapped_var2 ).astype(int)\n",
    "    \n",
    "    # Source\n",
    "source_counts = data.Source.value_counts()\n",
    "major_source = list(source_counts.index[:7])\n",
    "combined.loc[ ~combined['Source'].isin(major_source), 'Source' ] = 'Non-major source'\n",
    "mapped_source = dictMap(major_source, 'Non-major source')\n",
    "combined['Source'] = combined['Source'].map( mapped_source ).astype(int)\n",
    "\n",
    "    # 其他几个特征编码\n",
    "combined.Monthly_Income = combined.Monthly_Income.apply(np.sqrt) \n",
    "    \n",
    "combined.Loan_Amount_Applied = combined.Loan_Amount_Applied.apply(np.sqrt)\n",
    "    \n",
    "combined.Existing_EMI = [np.power(x, (float(1)/3)) for x in combined.Existing_EMI ] \n",
    "    \n",
    "combined.Loan_Amount_Submitted = combined.Loan_Amount_Submitted.apply(np.sqrt)\n",
    "    \n",
    "combined.DOB_yr = [np.log(x + 1) for x in combined.DOB_yr ] \n",
    "    \n",
    "combined.Processing_Fee = combined.Processing_Fee.apply(np.sqrt)\n",
    "    \n",
    "\n",
    "    \n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#combined =combined.fillna(9999,inplace=True)\n",
    "combined['EMI_Loan_Submitted'] = combined['EMI_Loan_Submitted'].replace('N',0) # 替换这一列中的一个错误值N"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\hplxg\\Anaconda3\\envs\\python3\\lib\\site-packages\\ipykernel_launcher.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "    # 把数据再分开\n",
    "data = combined.loc[ combined.Disbursed != 9999 ]\n",
    "test = combined.loc[ combined.Disbursed == 9999 ]\n",
    "    \n",
    "# 将目标变量再从测试集中去除\n",
    "test.drop(['LoggedIn','Disbursed'], axis=1, inplace=True)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\hplxg\\Anaconda3\\envs\\python3\\lib\\site-packages\\ipykernel_launcher.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  after removing the cwd from sys.path.\n"
     ]
    }
   ],
   "source": [
    "# data['is_train'] = np.random.uniform(0, 1, len(data)) <= .80\n",
    "# train, validate = data[data['is_train']==True], data[data['is_train']==False]\n",
    "train = data.drop([ 'LoggedIn'], axis=1)\n",
    "data['is_train'] = np.random.uniform(0, 1, len(data)) <= .80\n",
    "validate = data[data['is_train']==False]\n",
    "    \n",
    "    # 重新设置特征\n",
    "features=[\n",
    "    #'DOB',\n",
    "    'DOB_yr', \n",
    "    'Lead_Creation_Date',\n",
    "    'Gender',\n",
    "    'City',\n",
    "    'Monthly_Income',\n",
    "    'Loan_Amount_Applied',\n",
    "    'Loan_Tenure_Applied',\n",
    "    'Existing_EMI',\n",
    "    'Employer_Name',\n",
    "    'Salary_Account',\n",
    "    'Mobile_Verified',\n",
    "    'Var5',\n",
    "    'Var1',\n",
    "    'Loan_Amount_Submitted',\n",
    "    'Loan_Tenure_Submitted',\n",
    "    'Interest_Rate',\n",
    "    'Processing_Fee',\n",
    "    'EMI_Loan_Submitted',\n",
    "    'Filled_Form',\n",
    "    'Device_Type',\n",
    "    'Var2',\n",
    "    'Source',\n",
    "    'Var4',\n",
    "#    'missingness'\n",
    "    ]\n",
    "    \n",
    "    # X and Y\n",
    "x = data[list(features)].values\n",
    "y = data['Disbursed'].values\n",
    "X_train = train[list(features)].values\n",
    "x_validate = validate[list(features)].values\n",
    "y_train = train['Disbursed'].values\n",
    "y_validate = validate['Disbursed'].values\n",
    "x_test = test[list(features)].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[7.590346945602565, 735733, 0, ..., 6, 7, 1],\n",
       "       [7.593877844605118, 735722, 1, ..., 6, 7, 3],\n",
       "       [7.591861714889934, 735737, 1, ..., 7, 4, 1],\n",
       "       ...,\n",
       "       [7.587310506022615, 735810, 1, ..., 6, 7, 3],\n",
       "       [7.589841512182657, 735810, 1, ..., 6, 7, 3],\n",
       "       [7.5953872788539725, 735810, 1, ..., 6, 7, 4]], dtype=object)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0., 0., 0., ..., 0., 0., 0.])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2 利用XGBoost建立模型，并对模型的超参数调优"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.1 默认参数下调优"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# prepare cross validation\n",
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "#直接调用xgboost内嵌的交叉验证（cv），可对连续的n_estimators参数进行快速交叉验证\n",
    "#而GridSearchCV只能对有限个参数进行交叉验证\n",
    "def modelfit(alg, X_train, y_train, cv_folds=None, early_stopping_rounds=10):\n",
    "    xgb_param = alg.get_xgb_params()\n",
    "    xgb_param['num_class'] = 23\n",
    "    \n",
    "    #直接调用xgboost，而非sklarn的wrapper类\n",
    "    xgtrain = xgb.DMatrix(X_train, label = y_train)\n",
    "        \n",
    "    cvresult = xgb.cv(xgb_param, xgtrain, num_boost_round=alg.get_params()['n_estimators'], folds =cv_folds,\n",
    "             metrics='mlogloss', early_stopping_rounds=early_stopping_rounds)\n",
    "  \n",
    "    cvresult.to_csv('1_nestimators.csv', index_label = 'n_estimators')\n",
    "    \n",
    "    #最佳参数n_estimators\n",
    "    n_estimators = cvresult.shape[0]\n",
    "    \n",
    "    # 采用交叉验证得到的最佳参数n_estimators，训练模型\n",
    "    alg.set_params(n_estimators = n_estimators)\n",
    "    alg.fit(X_train, y_train, eval_metric='mlogloss')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "ename": "XGBoostError",
     "evalue": "b'value 0 for Parameter num_class should be greater equal to 1'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mXGBoostError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-20-e317a6657f64>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     12\u001b[0m         seed=3)\n\u001b[0;32m     13\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 14\u001b[1;33m \u001b[0mmodelfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mxgb1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcv_folds\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mkfold\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-19-4d34e48aeb9c>\u001b[0m in \u001b[0;36mmodelfit\u001b[1;34m(alg, X_train, y_train, cv_folds, early_stopping_rounds)\u001b[0m\n\u001b[0;32m     18\u001b[0m     \u001b[1;31m# 采用交叉验证得到的最佳参数n_estimators，训练模型\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     19\u001b[0m     \u001b[0malg\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mset_params\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mn_estimators\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mn_estimators\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 20\u001b[1;33m     \u001b[0malg\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0meval_metric\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'mlogloss'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\Anaconda3\\envs\\python3\\lib\\site-packages\\xgboost-0.81-py3.6.egg\\xgboost\\sklearn.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, X, y, sample_weight, eval_set, eval_metric, early_stopping_rounds, verbose, xgb_model, sample_weight_eval_set, callbacks)\u001b[0m\n\u001b[0;32m    698\u001b[0m                               \u001b[0mevals_result\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mevals_result\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeval\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mfeval\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    699\u001b[0m                               \u001b[0mverbose_eval\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mverbose\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mxgb_model\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 700\u001b[1;33m                               callbacks=callbacks)\n\u001b[0m\u001b[0;32m    701\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    702\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mobjective\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mxgb_options\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"objective\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\python3\\lib\\site-packages\\xgboost-0.81-py3.6.egg\\xgboost\\training.py\u001b[0m in \u001b[0;36mtrain\u001b[1;34m(params, dtrain, num_boost_round, evals, obj, feval, maximize, early_stopping_rounds, evals_result, verbose_eval, xgb_model, callbacks, learning_rates)\u001b[0m\n\u001b[0;32m    214\u001b[0m                            \u001b[0mevals\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mevals\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    215\u001b[0m                            \u001b[0mobj\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeval\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mfeval\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 216\u001b[1;33m                            xgb_model=xgb_model, callbacks=callbacks)\n\u001b[0m\u001b[0;32m    217\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    218\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\python3\\lib\\site-packages\\xgboost-0.81-py3.6.egg\\xgboost\\training.py\u001b[0m in \u001b[0;36m_train_internal\u001b[1;34m(params, dtrain, num_boost_round, evals, obj, feval, xgb_model, callbacks)\u001b[0m\n\u001b[0;32m     72\u001b[0m         \u001b[1;31m# Skip the first update if it is a recovery step.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     73\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mversion\u001b[0m \u001b[1;33m%\u001b[0m \u001b[1;36m2\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 74\u001b[1;33m             \u001b[0mbst\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdtrain\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mi\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     75\u001b[0m             \u001b[0mbst\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msave_rabit_checkpoint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     76\u001b[0m             \u001b[0mversion\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\python3\\lib\\site-packages\\xgboost-0.81-py3.6.egg\\xgboost\\core.py\u001b[0m in \u001b[0;36mupdate\u001b[1;34m(self, dtrain, iteration, fobj)\u001b[0m\n\u001b[0;32m   1043\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mfobj\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1044\u001b[0m             _check_call(_LIB.XGBoosterUpdateOneIter(self.handle, ctypes.c_int(iteration),\n\u001b[1;32m-> 1045\u001b[1;33m                                                     dtrain.handle))\n\u001b[0m\u001b[0;32m   1046\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1047\u001b[0m             \u001b[0mpred\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdtrain\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\python3\\lib\\site-packages\\xgboost-0.81-py3.6.egg\\xgboost\\core.py\u001b[0m in \u001b[0;36m_check_call\u001b[1;34m(ret)\u001b[0m\n\u001b[0;32m    163\u001b[0m     \"\"\"\n\u001b[0;32m    164\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mret\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 165\u001b[1;33m         \u001b[1;32mraise\u001b[0m \u001b[0mXGBoostError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0m_LIB\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mXGBGetLastError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    166\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    167\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mXGBoostError\u001b[0m: b'value 0 for Parameter num_class should be greater equal to 1'"
     ]
    }
   ],
   "source": [
    "#params = {\"objective\": \"multi:softprob\", \"eval_metric\":\"mlogloss\", \"num_class\": 9}\n",
    "xgb1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=1000,  #数值大没关系，cv会自动返回合适的n_estimators\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel=0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "modelfit(xgb1, X_train, y_train, cv_folds = kfold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cvresult = pd.DataFrame.from_csv('1_nestimators.csv')\n",
    "        \n",
    "# plot\n",
    "test_means = cvresult['test-mlogloss-mean']\n",
    "test_stds = cvresult['test-mlogloss-std'] \n",
    "        \n",
    "train_means = cvresult['train-mlogloss-mean']\n",
    "train_stds = cvresult['train-mlogloss-std'] \n",
    "\n",
    "x_axis = range(0, cvresult.shape[0])\n",
    "        \n",
    "pyplot.errorbar(x_axis, test_means, yerr=test_stds ,label='Test')\n",
    "pyplot.errorbar(x_axis, train_means, yerr=train_stds ,label='Train')\n",
    "pyplot.title(\"XGBoost n_estimators vs Log Loss\")\n",
    "pyplot.xlabel( 'n_estimators' )\n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig( 'n_estimators4_1.png' )\n",
    "\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cvresult = pd.DataFrame.from_csv('1_nestimators.csv')\n",
    "\n",
    "cvresult = cvresult.iloc[100:]\n",
    "# plot\n",
    "test_means = cvresult['test-mlogloss-mean']\n",
    "test_stds = cvresult['test-mlogloss-std'] \n",
    "        \n",
    "train_means = cvresult['train-mlogloss-mean']\n",
    "train_stds = cvresult['train-mlogloss-std'] \n",
    "\n",
    "x_axis = range(100,cvresult.shape[0]+100)\n",
    "        \n",
    "fig = pyplot.figure(figsize=(10, 10), dpi=100)\n",
    "pyplot.errorbar(x_axis, test_means, yerr=test_stds ,label='Test')\n",
    "pyplot.errorbar(x_axis, train_means, yerr=train_stds ,label='Train')\n",
    "pyplot.title(\"XGBoost n_estimators vs Log Loss\")\n",
    "pyplot.xlabel( 'n_estimators' )\n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig( 'n_estimators_detail.png' )\n",
    "\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.2 对TreeDepth和ChildWeight两个参数调优"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#max_depth 建议3-10， min_child_weight=1／sqrt(ratio_rare_event) =5.5\n",
    "max_depth = range(3,10,2)\n",
    "min_child_weight = range(1,6,2)\n",
    "param_test2_1 = dict(max_depth=max_depth, min_child_weight=min_child_weight)\n",
    "param_test2_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb2_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=699,  #第一轮参数调整得到的n_estimators最优值\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch2_1 = GridSearchCV(xgb2_1, param_grid = param_test2_1, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch2_1.fit(X_train , y_train)\n",
    "\n",
    "gsearch2_1.grid_scores_, gsearch2_1.best_params_,     gsearch2_1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gsearch2_1.grid_scores_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch2_1.best_score_, gsearch2_1.best_params_))\n",
    "test_means = gsearch2_1.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch2_1.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch2_1.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch2_1.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch2_1.cv_results_).to_csv('my_preds_maxdepth_min_child_weights_1.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(max_depth), len(min_child_weight))\n",
    "train_scores = np.array(train_means).reshape(len(max_depth), len(min_child_weight))\n",
    "\n",
    "for i, value in enumerate(max_depth):\n",
    "    pyplot.plot(min_child_weight, -test_scores[i], label= 'test_max_depth:'   + str(value))\n",
    "#for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'max_depth' )                                                                                                      \n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig('max_depth_vs_min_child_weght_1.png' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#max_depth 建议3-10， min_child_weight=1／sqrt(ratio_rare_event) =5.5\n",
    "max_depth = [6,7,8]\n",
    "min_child_weight = [4,5,6]\n",
    "param_test2_2 = dict(max_depth=max_depth, min_child_weight=min_child_weight)\n",
    "param_test2_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb2_2 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=699,  #第一轮参数调整得到的n_estimators最优值\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch2_2 = GridSearchCV(xgb2_2, param_grid = param_test2_2, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch2_2.fit(X_train , y_train)\n",
    "\n",
    "gsearch2_2.grid_scores_, gsearch2_2.best_params_,     gsearch2_2.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gsearch2_2.grid_scores_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch1.best_score_, gsearch1.best_params_))\n",
    "test_means = gsearch2_2.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch2_2.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch2_2.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch2_2.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch2_2.cv_results_).to_csv('my_preds_maxdepth_min_child_weights_2.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(min_child_weight), len(max_depth))\n",
    "train_scores = np.array(train_means).reshape(len(min_child_weight), len(max_depth))\n",
    "\n",
    "for i, value in enumerate(min_child_weight):\n",
    "    pyplot.plot(max_depth, test_scores[i], label= 'test_min_child_weight:'   + str(value))\n",
    "#for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'max_depth' )                                                                                                      \n",
    "pyplot.ylabel( '- Log Loss' )\n",
    "pyplot.savefig( 'max_depth_vs_min_child_weght2.png' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
